package io.github.biezhi.elves.spider;

import java.io.File;
import java.io.FileInputStream;
import java.io.FileNotFoundException;
import java.io.IOException;
import java.io.InputStreamReader;
import java.net.InetSocketAddress;
import java.net.Proxy;
import java.net.SocketTimeoutException;
import java.text.SimpleDateFormat;
import java.util.ArrayList;
import java.util.Date;
import java.util.HashMap;
import java.util.List;
import java.util.ListIterator;
import java.util.Locale;
import java.util.Map;
import java.util.regex.Matcher;
import java.util.regex.Pattern;

import org.jsoup.Jsoup;
import org.jsoup.nodes.Document;
import org.jsoup.nodes.Element;
import org.jsoup.select.Elements;
import org.springframework.beans.factory.annotation.Autowired;
import org.springframework.stereotype.Component;

import com.alibaba.fastjson.JSONObject;

import io.github.biezhi.elves.bidding.model.DownloadFileDTO;
import io.github.biezhi.elves.common.Constants;
import io.github.biezhi.elves.common.FileType;
import io.github.biezhi.elves.ok.http.FastHttpClient;
import io.github.biezhi.elves.service.JmggzyService;
import io.github.biezhi.elves.jmggzy.model.JmBiddingList;
import io.github.biezhi.elves.jmggzy.model.JmDownloadBiddingHtml;
import io.github.biezhi.elves.jmggzy.model.JmRecordBiddingNotice;
import io.github.biezhi.elves.utils.StringUtils;
/**
 * 韶关公共资源交易网爬取(http://www.fjmgzy.cn/)
 * @author Administrator
 *
 */
@Component
public class JmggzySpider {
	
	@Autowired
	private JmggzyService jmggzyService;
	
	//休眠时间
	Long sleepTime = 1000L;
	//url
	private String url = "http://zyjy.jiangmen.gov.cn";
	//页码，默认第一页
	Integer pageIndex = 1;
	//保存所有关键字信息到数据库
	public void saveJmRecord() throws Exception{
		String keyword = "";
		Integer lastPageIndex = getLastPageIndexByKeyword(keyword);
		saveJmRecordByOnekey(keyword,lastPageIndex);
	}
	//根据一个关键字保存JmDownloadBiddingHtml对象
	@SuppressWarnings("deprecation")
	private void saveJmRecordByOnekey(String keyword, Integer lastPageIndex) throws Exception {

		for (pageIndex = 1; pageIndex <= lastPageIndex; pageIndex++) {
			//获取整个网页信息
			String htmlContent = getHtmlContent(keyword,pageIndex);
			if (htmlContent != null && htmlContent.length()>0) {
			//解析文档
			Document doc = Jsoup.parse(htmlContent);
			Elements elements = doc.select("div.tab-item.itemtw > ul > li");
			
			for (Element element : elements) {
				JmRecordBiddingNotice jmRecordBiddingNotice = new JmRecordBiddingNotice();
				//获取发布日期
				String pubdate="";
				//获取公告标题
				String title="";
				//获取公告详情链接
				String noticeDetailUrl="";
				//获取公告类型
				String noticeType="";
				//获取标签类型
				String tabType="";
				
				pubdate = element.select("a > span.date").text();
				SimpleDateFormat sdf = new SimpleDateFormat("yyyy-MM-dd HH:mm:ss");
				Date date = sdf.parse(pubdate);
				SimpleDateFormat sdf2 = new SimpleDateFormat("yyyy-MM-dd");
				pubdate = sdf2.format(date);
				noticeDetailUrl = element.select("a").attr("href");
				title = element.select("a").attr("title");
				noticeType = "结果公告";
				tabType = "政府采购";
				
				jmRecordBiddingNotice.setNoticeType(noticeType);
				jmRecordBiddingNotice.setKeyword(keyword);
				jmRecordBiddingNotice.setNoticeDetailUrl(noticeDetailUrl);
				jmRecordBiddingNotice.setPubdate(pubdate);
				jmRecordBiddingNotice.setSize(pageIndex);
				jmRecordBiddingNotice.setTitle(title);
				jmRecordBiddingNotice.setTabType(tabType);
				
				//根据数据库是否有这个noticeDetailUrl来判断条信息是否已经保存过，保存过就不再保存了
				Integer count = jmggzyService.countByUrl(noticeDetailUrl);
				if (count == 0) {
					//保存fsRecordBiddingNotice到数据库
					jmggzyService.saveJmRecordBiddingNotice(jmRecordBiddingNotice);
				}
			}
			}
			System.out.println(pageIndex);
			if (pageIndex==lastPageIndex) {
				System.out.println(keyword+"关键字执行完成！");
			}
		}
	
	}

	private Integer getLastPageIndexByKeyword(String keyword) {
		String htmlContent = getHtmlContent(keyword, 1);
		//System.out.println(htmlContent);
		if (htmlContent != null && htmlContent.length()>0) {
			//解析文档
			Document doc = Jsoup.parse(htmlContent);
			String mainText = doc.select(".pagesite").text();
//			System.out.println(mainText);
			String lastPage = "";
			int totalPage = 1;
			String utilRegex = "/[\\s\\S]+页  首页";
			Pattern pattern = Pattern.compile(utilRegex);
			Matcher matcher = pattern.matcher(mainText);
			while (matcher.find()) {
				lastPage = matcher.group();
				lastPage = lastPage.replace("/", "");
				lastPage = lastPage.replace("页  首页", "");
			}
			totalPage = Integer.valueOf(lastPage);
			System.out.println("总页数："+totalPage);
			return totalPage;
		}
		return 1;
	}
	
	//读取record表中的数据，根据notice_detail_url查询公告内容，并保存到t_download_jm
	public void saveJmDownload(){
		//查询record表中notice_type等于中标公告并且发布日期在2018年的数据
		String noticeType = "结果公告";
		String date = "2018";
		
		List<JmRecordBiddingNotice> listJmRecordBiddingNotice = jmggzyService.findRecordByNoticeTypeAndDate(noticeType,date);
		
		//查询出符合条件所有的notice_detail_url
		for (JmRecordBiddingNotice jmRecordBiddingNotice : listJmRecordBiddingNotice) {
			//定义一个JmDownloadBiddingHtml对象，将数据存入对象
			JmDownloadBiddingHtml JmDownloadBiddingHtml = new JmDownloadBiddingHtml();
			
			String noticeDetailUrl = jmRecordBiddingNotice.getNoticeDetailUrl();
			if (noticeDetailUrl!=null && noticeDetailUrl.length()>0) {
				String htmlContent = getHtmlContent(noticeDetailUrl);
				String keyword = jmRecordBiddingNotice.getKeyword();
				String pubdate = jmRecordBiddingNotice.getPubdate();
				String tabType = jmRecordBiddingNotice.getTabType();
				
				JmDownloadBiddingHtml.setHtmlContent(htmlContent);
				JmDownloadBiddingHtml.setKeyword(keyword);
				JmDownloadBiddingHtml.setNoticeDetailUrl(noticeDetailUrl);
				JmDownloadBiddingHtml.setPubdate(pubdate);
				JmDownloadBiddingHtml.setNoticeType("结果公告");
				JmDownloadBiddingHtml.setTabType(tabType);
				
				Document doc = Jsoup.parse(htmlContent);
				//查找正文中所有a标签
				Elements allAElements = doc.select("a");
				ListIterator<Element> htmlAResultIter = allAElements.listIterator();
				
				List<DownloadFileDTO> listDownloadFile = new ArrayList<DownloadFileDTO>();
				while(htmlAResultIter.hasNext()) {
					Element get = htmlAResultIter.next();
					String fileName = get.text();
					String fileUrl = get.attr("href");
					//文件后缀
					String[] fileSuffixs = FileType.FILE_SUFFIX.split(",");
					for(String fileSuffix : fileSuffixs) {
						//判断获取的文件链接与文件后缀是否匹配
						if(fileUrl.endsWith(fileSuffix)) {
							DownloadFileDTO downloadFile = new DownloadFileDTO();
							downloadFile.setFileName(fileName);
							downloadFile.setFileUrl(fileUrl);
							
							System.out.println("fileName：" + fileName);
							System.out.println("fileUrl：" + fileUrl);
							listDownloadFile.add(downloadFile);
							break;
						}
					}
					JmDownloadBiddingHtml.setDownloadFile(JSONObject.toJSONString(listDownloadFile));
				}
				
				
				//查询这条公告在数据库是否存在
				Integer count = jmggzyService.countByDownloadUrl(noticeDetailUrl);
				if (count==0) {
					//保存JmDownloadBiddingHtml对象到t_download_jm表中
					jmggzyService.saveJmDownloadBiddingHtml(JmDownloadBiddingHtml);
				}
			}
			
		}
		System.out.println("插入完成！");
	}
	
	//解析文档
	public void resolveHtmlContent(){
		//读取数据库 获取文档信息
		//List<JmDownloadBiddingHtml> listJmDownloadBiddingHtml = jmggzyService.listDownloadHtml();
		//获取tab_type='政府采购',的JmDownloadBiddingHtml
		String tabType = "政府采购";
		List<JmDownloadBiddingHtml> listJmDownloadBiddingHtml = jmggzyService.listDownloadHtmlByTabType(tabType);
		if (!listJmDownloadBiddingHtml.isEmpty()) {
			for (JmDownloadBiddingHtml jmDownloadBiddingHtml : listJmDownloadBiddingHtml) {
				String htmlContent = jmDownloadBiddingHtml.getHtmlContent();
				String pubdate = jmDownloadBiddingHtml.getPubdate();
				String noticeDetailUrl = jmDownloadBiddingHtml.getNoticeDetailUrl();
				if (htmlContent != null && !htmlContent.equals("")) {
					//解析htmlContent,返回JmbiddingList对象
					JmBiddingList jmbiddingList = resolveToJmBiddingList(htmlContent);
					//公告详情内容链接存入对象
					jmbiddingList.setBidUrl(noticeDetailUrl);
					//发布时间存入对象
					jmbiddingList.setPubdate(pubdate);
					//插入到数据库
					jmggzyService.saveJmBiddingList(jmbiddingList);
				}
			}
		}
		System.out.println("文档字段插入完成！");
	}

	private JmBiddingList resolveToJmBiddingList(String htmlContent) {
		JmBiddingList jmBiddingList = new JmBiddingList();
//		htmlContent = "";
//		htmlContent = getXmlString();
		Document doc = Jsoup.parse(htmlContent);
		//获取公告标题
		String noticeTitle = doc.select("h1").text();
		System.out.println("标题："+noticeTitle);
		jmBiddingList.setNoticeTitle(noticeTitle);
		//城市
		String city = "江门";
		System.out.println("城市："+city);
		jmBiddingList.setCity(city);
		
		if (doc.select("div.newsCon") !=null) {
			String content2 = doc.select("div.newsCon").text();
			System.out.println(content2);
			//项目id
			String prjId = "";
			String utilRegex = "一、采购项目编号：[\\s\\S]+二、采购项目名称";
			Pattern pattern = Pattern.compile(utilRegex);
			Matcher matcher = pattern.matcher(content2);
			while (matcher.find()) {
				prjId = matcher.group();
				prjId = prjId.replace("一、采购项目编号：", "");
				prjId = prjId.replace("二、采购项目名称", "");
			}
			if (prjId.length()>100) {
				prjId="";
			}
			System.out.println("项目ID："+prjId);
			jmBiddingList.setPrjId(prjId);
			//供应商
			String supplier = "";
			String utilRegex2 = "供应商名称：[\\s\\S]+2、中标供应商地址：";
			Pattern pattern2 = Pattern.compile(utilRegex2);
			Matcher matcher2 = pattern2.matcher(content2);
			while (matcher2.find()) {
				supplier = matcher2.group();
				supplier = supplier.replace("供应商名称：", "");
				supplier = supplier.replace("2、中标供应商地址：", "");
			}
			if (StringUtils.isNullOrEmpty(supplier)) {
				utilRegex2 = "中标供应商名称:[\\s\\S]+法人代表";
				pattern2 = Pattern.compile(utilRegex2);
				matcher2 = pattern2.matcher(content2);
				while (matcher2.find()) {
					supplier = matcher2.group();
					supplier = supplier.replace("中标供应商名称:", "");
					supplier = supplier.replace("法人代表", "");
				}
			}
			if (StringUtils.isNullOrEmpty(supplier)) {
				utilRegex2 = "中标供应商名称：[\\s\\S]+2.中标供应商地址";
				pattern2 = Pattern.compile(utilRegex2);
				matcher2 = pattern2.matcher(content2);
				while (matcher2.find()) {
					supplier = matcher2.group();
					supplier = supplier.replace("中标供应商名称：", "");
					supplier = supplier.replace("2.中标供应商地址", "");
				}
			}
			System.out.println("供应商:"+supplier);
			if (supplier.length()>100) {
				supplier="";
			}
			jmBiddingList.setSupplier(supplier);
			//项目名称
			String prjName = "";
			String utilRegex3 = "二、采购项目名称：[\\s\\S]+三、采购项目预算金额";
			Pattern pattern3 = Pattern.compile(utilRegex3);
			Matcher matcher3 = pattern3.matcher(content2);
			while (matcher3.find()) {
				prjName = matcher3.group();
				prjName = prjName.replace("二、采购项目名称：", "");
				prjName = prjName.replace("三、采购项目预算金额", "");
			}
			if (prjName.length()>100) {
				prjName="";
			}
			System.out.println("项目名称："+prjName);
			jmBiddingList.setPrjName(prjName);
			//中标时间
			String dealTime = "";
			String utilRegex4 = "评审日期[\\s\\S]+评审地点";
			Pattern pattern4 = Pattern.compile(utilRegex4);
			Matcher matcher4 = pattern4.matcher(content2);
			while (matcher4.find()) {
				dealTime = matcher4.group();
				dealTime = dealTime.replace("评审日期", "");
				dealTime = dealTime.replace("评审地点", "");
			}
			if (dealTime.length()>100) {
				dealTime="";
			}
			System.out.println("中标时间:"+dealTime);
			jmBiddingList.setDealTime(dealTime);
			//预算金额
			String budget = "";
			String utilRegex5 = "预算金额（元）：[\\s\\S]+四、采购方式";
			Pattern pattern5 = Pattern.compile(utilRegex5);
			Matcher matcher5 = pattern5.matcher(content2);
			while (matcher5.find()) {
				budget = matcher5.group();
				budget = budget.replace("预算金额（元）：", "");
				budget = budget.replace("四、采购方式", "");
			}
			System.out.println("预算金额："+budget);
			jmBiddingList.setBudget(budget);
			//采购方式
			String buyType = "";
			String utilRegex6 = "四、采购方式：[\\s\\S]+五、";
			Pattern pattern6 = Pattern.compile(utilRegex6);
			Matcher matcher6 = pattern6.matcher(content2);
			while (matcher6.find()) {
				buyType = matcher6.group();
				buyType = buyType.replace("四、采购方式：", "");
				buyType = buyType.replace("五、", "");
			}
			System.out.println("采购方式："+buyType);
			jmBiddingList.setBuyType(buyType);
			//获取客户（发标、采购方）
			String customer = "";
			String utilRegex7 = "受[\\s\\S]+的委托";
			Pattern pattern7 = Pattern.compile(utilRegex7);
			Matcher matcher7 = pattern7.matcher(content2);
			while (matcher7.find()) {
				customer = matcher7.group();
				customer = customer.replace("受", "");
				customer = customer.replace("的委托", "");
			}
			if (customer.length()>100) {
				customer="";
			}
			System.out.println("客户（发标、采购方）："+customer);
			jmBiddingList.setCustomer(customer);
			//发标时间
			String publishTime = "";
			String utilRegex8 = "于[\\s\\S]+日就";
			Pattern pattern8 = Pattern.compile(utilRegex8);
			Matcher matcher8 = pattern8.matcher(content2);
			while (matcher8.find()) {
				publishTime = matcher8.group();
				publishTime = publishTime.replace("于", "");
				publishTime = publishTime.replace("就", "");
			}
			if (publishTime.length()>100) {
				publishTime="";
			}
			System.out.println("发标时间:"+publishTime);
			jmBiddingList.setPublishTime(publishTime);
		}
		//成交额
		/*if (doc.select(".MsoTableGrid tr") != null && doc.select(".MsoTableGrid tr").size()>1 && 
				doc.select(".MsoTableGrid tr").get(1).select("td")!=null && 
						doc.select(".MsoTableGrid tr").get(1).select("td").size()>5) {
			
			String bidPrice = doc.select(".MsoTableGrid tr").get(1).select("td").get(5).text();
		System.out.println("成交额:"+bidPrice);
			jmBiddingList.setBidPrice(bidPrice);
		}*/
		if (doc.select("table") != null && doc.select("table").size()>0 
				&& doc.select("table").get(0).select("tr") != null 
				&& doc.select("table").get(0).select("tr").size()>1 
				&& doc.select("table").get(0).select("tr").get(1).select("td") != null
				&& doc.select("table").get(0).select("tr").get(1).select("td").size() >5) {
			String bidPrice = doc.select("table").get(0).select("tr").get(1).select("td").get(5).text();
			if (bidPrice.length()>100) {
				bidPrice="";
			}
			System.out.println("成交额:"+bidPrice);
			jmBiddingList.setBidPrice(bidPrice);
		}
		return jmBiddingList;
	}
	//post方式爬取网页内容
	private String postHtmlContent(String keyword, Integer pageIndex) {
		String result = "";
		String bidUrl = "http://www.jmjyzx.com/textSearchAction!frontTextSearchListChildren2.do";
		try {
			Map<String, String> paramMap = new HashMap<String, String>();
			//paramMap.put("textSearch.title", keyword);
			paramMap.put("page", pageIndex.toString());
			paramMap.put("pageSize", "20");
			paramMap.put("sortField", "CREATETIME");
			paramMap.put("sortOrder", "DESC");
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
			okHttpResponse = FastHttpClient
					.newBuilder()
					.proxy(proxy)
					.build()
					.post()
					.addParams(paramMap)
					.url(bidUrl)
					.build()
					.execute();
		
			result = okHttpResponse.string();
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用postHtmlContent()方法===");
			postHtmlContent(keyword, pageIndex);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用postHtmlContent()方法===");
			postHtmlContent(keyword, pageIndex);
		}
		return result;
	}
	//get方式爬取网页内容
	private String getHtmlContent(String url) {
		String result = "";
		try {
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
			okHttpResponse = FastHttpClient
					.newBuilder()
					.proxy(proxy)
					.build()
					.get()
					.url(url)
					.build()
					.execute();
			
			result = okHttpResponse.string();
			//System.out.println(result);
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用getHtmlContent()方法===");
			getHtmlContent(url);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用getHtmlContent()方法===");
			getHtmlContent(url);
		}
		return result;
	}
	
	private String getHtmlContent(String keyword, Integer pageIndex) {
		String bidUrl = "http://zyjy.jiangmen.gov.cn/cjgg/index.htm";
		if (pageIndex > 1) {
			bidUrl = "http://zyjy.jiangmen.gov.cn/cjgg/index_"+pageIndex+".htm";
		}
		String result = "";
		try {
			io.github.biezhi.elves.ok.http.Response okHttpResponse;
			Proxy proxy = new Proxy(Proxy.Type.HTTP,new InetSocketAddress(Constants.PROXY_IP, Constants.PROXY_PORT));
			okHttpResponse = FastHttpClient
					.newBuilder()
					.proxy(proxy)
					.build()
					.get()
					.addParams("channelId", "104")
					.url(bidUrl)
					.build()
					.execute();
			
			result = okHttpResponse.string();
			//System.out.println(result);
			if(!StringUtils.isNullOrEmpty(result)){
				return result;
			}
		} catch (SocketTimeoutException e) {
			System.err.println("===连接超时，重新调用getHtmlContent()方法===");
			getHtmlContent(url);
		} catch (Exception e) {
			e.printStackTrace();
			System.err.println("===其他异常，重新调用getHtmlContent()方法===");
			getHtmlContent(url);
		}
		return result;
	}
	
	//读取文本为字符串
	public String getXmlString() {
		String xmlString;
		char[] strBuffer = null;
		int flen = 0;
		File xmlfile = new File("D:\\jmHtml.txt");
		try {
		FileInputStream input = new FileInputStream(xmlfile);
		InputStreamReader in=new InputStreamReader(input,"utf-8");//字节流字符流转化的桥梁
		flen = (int)xmlfile.length();
		strBuffer = new char[flen];
		in.read(strBuffer, 0, flen);
		} catch (FileNotFoundException e) {
		e.printStackTrace();
		} catch (IOException e) {
		e.printStackTrace();
		}
		xmlString = new String(strBuffer); //构建String时，可用char[]类型，

		return xmlString;
		}
	
	public static void main(String[] args) throws Exception {
		JmggzySpider ss = new JmggzySpider();
//		Integer lastPageIndexByKeyword = ss.getLastPageIndexByKeyword("");
		JmBiddingList resolveToJmBiddingList = ss.resolveToJmBiddingList("xxx");
//		ss.saveJmRecordByOnekey("", 1);
	}
}
